{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "9c84df67-5ecc-4aa5-9eea-8776d0031245",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "github_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/04-monitoring/data/results-gpt4o-mini.csv'\n",
    "url = f'{github_url}?raw=1'\n",
    "df = pd.read_csv(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "3d213c24-f3e7-47d6-b6fa-7e019d84e6c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.iloc[:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "30c87dec-1f75-4b09-a702-aff863e19226",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33012c674aca48f281543b47de5652ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "modules.json:   0%|          | 0.00/229 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alexe\\miniconda3\\Lib\\site-packages\\huggingface_hub\\file_download.py:157: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\alexe\\.cache\\huggingface\\hub\\models--sentence-transformers--multi-qa-mpnet-base-dot-v1. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80396d6ac78a4487a5382ade93a08b2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config_sentence_transformers.json:   0%|          | 0.00/212 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You try to use a model that was created with version 3.0.0.dev0, however, your version is 2.7.0. This might cause unexpected behavior or errors. In that case, try to update to the latest version.\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1bf41cb37824d4c89c8af56ce252eaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/8.71k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48d6c3a25b0043d1a5541f48525422bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentence_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6761ab598f6240ec885ae57b511ab1e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/571 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "000091ce6f744f9baf5c492cb46d130d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/438M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cba0092b243942e1ac0f6739b3679830",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/363 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3979308f035f41e08a68569003b0860d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51c905c793d74ac4a829235d76d01fb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5466fece4ce4f9ab22684e71523c7bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/239 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13c09e2413a04326bc5a0209415783cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "1_Pooling/config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "model_name = 'multi-qa-mpnet-base-dot-v1'\n",
    "embedding_model = SentenceTransformer(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ce2a3af-d081-43b2-a8ec-dfc8073b7525",
   "metadata": {},
   "outputs": [],
   "source": [
    "q = df.iloc[0].answer_llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "990885f2-b5e2-4c97-b453-bdd99b716ce1",
   "metadata": {},
   "outputs": [],
   "source": [
    "v = embedding_model.encode(q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "313edb66-4b91-48e3-ab70-fa88492bb66e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.4224466"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "c7a27899-706f-4c95-9845-5735152b5985",
   "metadata": {},
   "outputs": [],
   "source": [
    "records = df.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "d27cfde8-111a-40d5-be2f-e16264f4285a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4c295044-73be-4447-b978-ac2d5e0447e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "866085d089b84c7889ec2333dd272318",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/300 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluations = []\n",
    "\n",
    "for r in tqdm(records):\n",
    "    u = embedding_model.encode(r['answer_llm'])\n",
    "    v = embedding_model.encode(r['answer_orig'])\n",
    "    evaluations.append(u.dot(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a024ae10-a35d-492f-8323-49838d591d24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    300.000000\n",
       "mean      27.495996\n",
       "std        6.384743\n",
       "min        4.547925\n",
       "25%       24.307845\n",
       "50%       28.336869\n",
       "75%       31.674311\n",
       "max       39.476021\n",
       "dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluations = pd.Series(evaluations)\n",
    "evaluations.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "18cbc773-a6ef-46df-bc31-9b375e95cede",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.947308"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.dot(u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d69cb48a-78f2-400d-ac31-d92110d11a3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4920f22f-dff8-4f12-a95a-dda9f8f0d77b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(v):\n",
    "    return v / np.sqrt((v * v).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "13595e4b-930f-4d39-87e3-38cce8b773a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000001"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vn.dot(vn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c50ff56a-fd17-4b81-b6a3-a49a0cab4308",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4afdb2575cca4ae2a6b14e0b37875512",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/300 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluations = []\n",
    "\n",
    "for r in tqdm(records):\n",
    "    u = embedding_model.encode(r['answer_llm'])\n",
    "    v = embedding_model.encode(r['answer_orig'])\n",
    "    u = normalize(u)\n",
    "    v = normalize(v)\n",
    "    evaluations.append(u.dot(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "6c61c147-e7db-4a6c-8fcd-d5158f80b738",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    300.000000\n",
       "mean       0.728392\n",
       "std        0.157755\n",
       "min        0.125357\n",
       "25%        0.651273\n",
       "50%        0.763761\n",
       "75%        0.836235\n",
       "max        0.958796\n",
       "dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluations = pd.Series(evaluations)\n",
    "evaluations.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "ddf5e709-a30b-48fd-86c4-191e98f23718",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rouge import Rouge\n",
    "rouge_scorer = Rouge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "75cbdafc-998a-434e-a9a2-346e28cc7a23",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = records[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "bfa8beb2-f9c5-4d2a-a8ed-3ff445dd1799",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = rouge_scorer.get_scores(r['answer_llm'], r['answer_orig'])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "d110ce2b-908b-4f4f-a304-4c2d82bf3e9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45454544954545456"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores['rouge-1']['f']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "19fac191-d95f-42bc-af30-1b155530e2fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge_1 = scores['rouge-1']['f']\n",
    "rouge_2 = scores['rouge-2']['f']\n",
    "rouge_l = scores['rouge-l']['f']\n",
    "rouge_avg = (rouge_1 + rouge_2 + rouge_l) / 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "585cd96b-d6be-4e76-90b8-4256690d9bff",
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge_avg = (rouge_1 + rouge_2 + rouge_l) / 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "90bc8f8c-70c2-40f3-b004-28f469f9bde6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35490034990035496"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rouge_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "2cef0852-a016-428c-bc7f-37daf1f058b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08347a15eac44c20941a861a5ed3459d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/300 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluations = []\n",
    "\n",
    "for r in tqdm(records):\n",
    "    scores = rouge_scorer.get_scores(r['answer_llm'], r['answer_orig'])[0]\n",
    "    \n",
    "    rouge_1 = scores['rouge-1']['f']\n",
    "    rouge_2 = scores['rouge-2']['f']\n",
    "    rouge_l = scores['rouge-l']['f']\n",
    "    \n",
    "    evaluations.append({\n",
    "        'rouge_1': rouge_1,\n",
    "        'rouge_2': rouge_2,\n",
    "        'rouge_l': rouge_l,\n",
    "        'rouge_avg': (rouge_1 + rouge_2 + rouge_l) / 3\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2bce2630-6682-4741-a772-19a1bc07c98e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20696501983423318"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(evaluations).rouge_2.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a5f90dc-8206-4e63-aa0a-b1f990baba90",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
